from httplib2 import Http
from urllib import urlencode

from BeautifulSoup import BeautifulSoup, MinimalSoup, BeautifulStoneSoup, ICantBelieveItsBeautifulSoup

import Image, locale, re, shelve, math, os, sys, logging, shutil

def configureLogging():
    logger = logging.getLogger()

    LOG_FILENAME = absolutepath + 'logs/log.out'
    logging.basicConfig(filename=LOG_FILENAME,
                        level=logging.DEBUG,
                        format="%(asctime)s%(levelname)s::%(message)s",
                        datefmt='%Y-%m-%d[%H:%M:%S]')
    
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(asctime)s%(levelname)s::%(message)s", '%Y-%m-%d[%H:%M:%S]')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

def deleteBadLines(name = "saved"):
    
    f = open(absolutepath + name + '.html', 'r')
    
    contents2 = ""
    
    all_lines = f.readlines()

    for line in all_lines[1:]:
        if (not line.lstrip().startswith("['<a")):
            contents2 += line

    return contents2

def saveListImages(listImages):
    d = shelve.open(absolutepath + "listImages")
    
    d["listImages"] = listImages
    
    d.close()

def saveTempHtml(html, name = "saved"):
    f = open(absolutepath + name + '.html', 'w')
    
    f.write(html)
    
    f.close()

def getSoupFromFile():
    f = open(absolutepath + 'saved.html', 'r')
    
    soup = BeautifulSoup (f.read())
    
    f.close()
    
    return soup

def getSoup(materialsearch):
    
    urlCircusMuseum =   "http://www.circusmuseum.nl"
    pathSearch      =   "/index.php?option=com_memorix&Itemid=26&task=result&searchplugin=quick"
    
    h = Http()
    
    headers = {'Content-type': 'application/x-www-form-urlencoded'} 
     
    body = {'materiaalsoort':materialsearch, 'trefwoord':''}
    
    response, content = h.request(urlCircusMuseum + pathSearch, "POST", headers=headers, body=urlencode(body))
    
    cookie = response['set-cookie']
    location = response['location']
    s1 = "PHPSESSID="
    phpsession = cookie[cookie.find(s1)+len(s1):cookie.find(";")]
                
    response, content = h.request(urlCircusMuseum + location, "GET")
    
    saveTempHtml(content)
    
    content2 = deleteBadLines()
    
    saveTempHtml(content2, "saved2")
    
    soup = MinimalSoup (content2)
    
    # get total pages
    foundPoints = soup.find(text=re.compile("\.\.\."))
    totalPages = locale.atoi(foundPoints.parent.nextSibling.next.string)
    
    return (urlCircusMuseum + location), totalPages

def getSoupFromPage(url, page):    
    h = Http()

    response, content = h.request(url + "&cp=" + str(page), "GET")
    
    saveTempHtml(content)
    
    content2 = deleteBadLines()
    
    soup = MinimalSoup (content2)
    
    return soup

def getListImages(soup, listImages):
    
    for id in soup.findAll('img', 'mrx_thumbnail_image'):
        pos = id['src'].find("thumbs/")+len("thumbs/")
        path = id['src'][pos:-4]
        listImages.append("http://www.circusmuseum.nl/cgi-bin/getpic?photo/topview/" + path + ".tjp")
        
    return listImages

def getListImagesFromFile():
    
    d = shelve.open(absolutepath + "listImages")
    
    listImages = d["listImages"]
    
    d.close()
        
    return listImages

class CircusImage:
    
    def __init__(self, width, height, layers, tilewidth, tileheight, ratio, numfiles, mimeType, url, page, name):
        self.width = locale.atoi(width)
        self.height = locale.atoi(height)
        self.layers = locale.atoi(layers)
        self.tilewidth = locale.atoi(tilewidth)
        self.tileheight = locale.atoi(tileheight) 
        self.ratio = locale.atoi(ratio) 
        self.numfiles = locale.atoi(numfiles)
        self.mimeType = mimeType
        self.url = url
        self.page = page
        self.name = name
        
        self.subdivwidth = int(math.ceil (float(self.width) / float(self.tilewidth)))
        self.subdivheight = int(math.ceil (float(self.height) / float(self.tileheight)))
        
        self.startPortions = int(self.numfiles - self.subdivwidth*self.subdivheight + 1)
        self.totalPortions = self.numfiles - self.startPortions 

    def saveImagePortions(self):

        dirpath = imagespath + '/portions/'
        
        if not os.path.exists(dirpath):
            os.makedirs(dirpath)
                  
        i = 0
        for subdiv in range(self.startPortions, self.numfiles + 1):
            h = Http()
            headers = {'Content-type': 'image/jpeg'}
            response, content = h.request(self.url + "&" + str(subdiv), "GET", headers=headers)
            
            f = open(dirpath + self.name + '.' + str(i) + '.jpg', 'wb')    
            f.write(content)            
            f.close()
            i += 1

    def mergePortions(self):
        im = Image.new("RGB", (self.width, self.height))
        box = (0, 0, self.tilewidth, self.tileheight)
        portionPath = imagespath + '/portions/' + self.name + '.'
        imagePath = imagespath + '/' + str(self.page) + '/'
        
        if not os.path.exists(imagePath):
            os.makedirs(imagePath)

        for h in range(self.subdivheight):
            for w in range(self.subdivwidth):
                imPortion = Image.open(portionPath + str(w + h*(self.subdivwidth)) + '.jpg')
                
                try:
                    im.paste(imPortion, box)
                except ValueError:
                    print "error:"
                    print imPortion
                    print box
                
                if (box[2] + self.tilewidth > self.width):
                    newWidth = self.width
                else:
                    newWidth = box[2] + self.tilewidth
                
                box = (box[0] + self.tilewidth, 
                       box[1], 
                       newWidth,
                       box[3])
                
            if (box[3] + self.tileheight > self.height):
                newHeight = self.height
            else:
                newHeight = box[3] + self.tileheight
                
            box = (0, 
                   box[1] + self.tileheight, 
                   self.tilewidth, 
                   newHeight)
        
        im.save(imagePath + self.name + '.jpg')

    def removePortions(self):
        shutil.rmtree(imagespath + '/portions/')

def saveImages(listImages, page):
    h = Http()
    
    imageNumber = 0
    for urlImage in listImages:
        imgname = re.findall("(?<=/TEY_[0-9]{3}/).*",urlImage)[0][:-4]
        
        logging.info ("exists " + imgname + "? " + str(os.path.exists(imagespath + '/' + str(page) + '/' + imgname + '.jpg')))
            
        if (os.path.exists(imagespath + '/' + str(page) + '/' + imgname + '.jpg')):
            
            logging.info ('pass')
            
        else:
                 
            response, content = h.request(urlImage, "GET")
            
            width, height, layers, tilewidth, tileheight, ratio, numfiles, mimeType = re.findall("(?<==).*", content)
    
            image = CircusImage(width, height, layers, tilewidth, tileheight, ratio, numfiles, mimeType, urlImage, page, imgname)
            
            logging.info("[%s] loaded image (%d x %d)", imgname, image.width, image.height)
            
            logging.info("[%s] start saving portions image %d/%d", imgname, imageNumber+1, len(listImages))        
            
            image.saveImagePortions()
            
            logging.info("[%s] total %d portions saved", imgname, image.totalPortions)  
            
            image.mergePortions()
            
            logging.info("[%s] portions merged. image created successfully", imgname)  
            
            image.removePortions()
            
        
        imageNumber += 1
        
        if imageNumber % 12 == 0:
            page += 1
    
def rename():    
    for i in range(100, 366):
        os.rename(imagespath + '/1/4732.3534.'+str(i)+'.jpg', imagespath + '/1/4732.3534.'+str(i-100)+'.jpg')
    
def main():    
    listImages = []
    materials = ['foto', 'affiche', 'prent']
    
    if (len(sys.argv) != 4):
        print ('usage: getimages <frompage> <topage> <>')
        print ('example: getimages 1 10 foto')
        return
    
    fromPage = int(sys.argv[1])
    toPage = int(sys.argv[2])
    category = sys.argv[3]
    
    if (not materials.__contains__(category)):        
        print ("category '%s' not recognized. Type: 'foto', 'affiches' or 'prenten'" % category)
        return
    
    global absolutepath
    global imagespath
    
    absolutepath = "/home/data/workspace/python/circusmuseum/"
    imagespath = absolutepath + "images/" + category
    
    configureLogging()
    
    logging.info("start process category %s from page %d to %d", category, fromPage, toPage)
    
    url, totalPages = getSoup(category)
    
    for page in range(fromPage, toPage+1):
        soup = getSoupFromPage(url, page)        
        listImages = getListImages(soup, listImages)
    
    saveListImages(listImages)
    
    # soup = getSoupFromFile()
    # listImages = getListImagesFromFile()
    
    logging.info("recovered %d url images successfully", len(listImages))
    
    saveImages (listImages, fromPage)
    
if __name__ == "__main__":
    main()